/*
 * Copyright (c) 2010, Internet Applications Research and Development, LLC.,
 * Stuart A MacKenzie
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification,  are  permitted  provided that the following conditions are met:
 * 1. Redistributions of source code must retain the above copyright notice, this
 * -  list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyrightnotice,
 * -  this list of conditions and the following disclaimer in the documentation
 * -  and / or other materials provided with the distribution.
 * 3. Neither the name of   Internet Applications Research and Development, LLC
 * -  nor the names of its contributors may be used to endorse or promote products
 * -  derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS  AND  CONTRIBUTORS  "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES,   INCLUDING,   BUT NOT LIMITED TO,  THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.   IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT,  INDIRECT,  INCIDENTAL,  SPECIAL, EXEMPLARY,  OR CONSEQUENTIAL
 * DAMAGES  (INCLUDING,  BUT NOT LIMITED TO,  PROCUREMENT  OF SUBSTITUTE GOODS OR
 * SERVICES;  LOSS OF USE,  DATA, OR PROFITS;  OR BUSINESS INTERRUPTION)  HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,  STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package net.iarad.util.csv;

import org.apache.log4j.Logger;

import java.io.*;
import java.util.ArrayList;
import java.util.List;

/**
 * Reads in a CSV file.  This counts the number of columns in the first
 * row of data returned.  It sets that count to be the column count.
 * The class checks to ensure that all subsequent rows contain the same
 * number of entries as the first number set.
 * <p>
 * If <code>setSkipErrorLine</code> is set to <code><i>true</i></code> (the default)
 * <code>CsvFileLoader</code> will attempt to skip the invalid row and proceed.  It
 * will log the error and log that it is attempting to continue.  If the
 * value is set to false, the file loader will throw a
 * <code>CsvException</code>.
 * <p>A comma <strong>can not</strong> appear at the end of a line, even if it
 * is followed by whitespace.  This will cause an error condition.  If
 * the last column in a row should be empty, it should be quoted:
 * <p>
 * example &nbsp; (<i>see row 2, column 3</i>):
 * <blockquote>
 * <code>
 * Row1_Col1, Row1_Col2, Row1_Col3<br>
 * Row2_Col1, Row2_Col2, ""
 * </code>
 * </blockquote>
 * <p>
 * The default delimiter is a <code>','</code> <i>comma</i>.  The delimiter can be
 * overridden using <code>setDelimiter</code>.  The comma does not need to escaped
 * inside of quotes.  The comma does need to be escaped when not in text
 * (examples below).
 * <p>
 * A comma can appear in either single or double quotes without being
 * escaped.</p>
 * <p>
 * The escape character is the <code>'\'</code> <i>backslash</i>.
 * <p>
 * A backslash <code>'\'</code> can be displayed by escaping it ( <code>\\</code> ).
 * It must be escaped both inside and outside of quotes.
 * <p>
 *
 * <strong>Examples:</strong>
 * <pre>
 * Escaping the comma outside of quotes:
 *   line...: Comma\, outside of quotes needs to be escaped, col2
 *   returns: [Comma, outside of quotes needs to be escaped] [col2]
 *
 * Comma inside quotes:
 *   line...: "Comma, in a quote does not need to be escaped", col2
 *   returns: [Comma, in a quote does not need to be escaped] [col2]
 *
 * Backslash inside and outside quotes:
 *   line...: "Backslash in a quote \\ needs to be escaped", col2
 *   returns: [Backslash in a quote \ needs to be escaped] [col2]
 *
 *   line...: Backslash \\ outside of quotes needs to be escaped, col2
 *   returns: [Backslash \ outside of quotes needs to be escaped] [col2]
 *
 *   010, 192.168.0.10, A, B, trailing delimiter (last position),""
 *
 * Quoted Text:
 *   line...: \"Quoted Text\", "col2"
 *   returns: ["Quoted Text"] [col2]
 *
 * Quoted Text with commas:
 *   line...: \"Quoted "with, some, commas," Text\","col2"
 *   returns: ["Quoted with, some, commas, Text"] [col2]
 *
 * More quoted text:
 *   line...: "Quoted, text is \"In Quotes\"", "col2"
 *   returns: [Quoted, text is "In Quotes"] [col2]
 *
 * <strong>Handling certain error conditions, end cases</strong>
 *
 * Trailing escape is ignored:
 *   case...: Col1, HANDLED ERROR - Start of escape in last position,\
 *   action.: System ignores trailing escape
 *
 * Trailing quote is ignored:
 *   case...: Col1, HANDLED ERROR - Start of quote in last position,"
 *   action.: System ignores trailing quote
 *
 * Quote is not closed and the string up to the ending position is returned
 *   case...: Col1, HANDLED ERROR - Quote not closed in last position,"Quote not closed
 *   action.: Quote is not closed and the string up to the trailing
 *            position is returned
 * </pre>
 *
 * @author Stuart A MacKenzie - defconx@gmail.com
 *
 * @version 0.09
 */
public class CsvReader {

    private static final Logger logger = Logger.getLogger(CsvReader.class);

    /**
     * Empty String array to return in place of <code>null</code>.
     */
    private static final String[] EMPTY_STRING_ARRAY = new String[0];

    /**
     * The delimiter to use for parsing the file.<br><br>
     * By default the comma (<code>,</code>) is the delimiter.
     */
    protected char delimiter = CHAR.DEFAULT_DELIMITER;

    //-- This is used so the filename string isn't null and throw
    //-- a null pointer exception when this class logs out exceptions.
    private static final String DEFAULT_FILENAME_STRING = "(Object) Reader";

    //-- File object associated with this class.  We use it
    //-- 	for convenience when printing out error messages
    //--	so we can refer to the filename.
    private String filename = DEFAULT_FILENAME_STRING;

    //-- Buffered Reader for reading CSV data
    //   TODO: can this be a Reader rather than a BufferedReader?
    private BufferedReader reader;

    //-- Try to recover by skipping an malformed row and continuing
    //-- 	instead of throwing and exception.
    private boolean skipErrorLines = true;

    //-- CAUTION - only set with setHeaderRow(String).
    private int columnCount = -1;


    //-- CAUTION - only set with incrementRowCount.  
    //-- The incrementRowCount() method will inc this too.
    //-- Includes blanks & comment lines
    private int lineCount = 0;

    //-- CAUTION - only set with incrementRowCount.
    //-- Only includes processed lines of data
    private int rowCount = 0;


    /**
     * Instantiates a CsvReader.
     * The user needs to pass in a file or a stream to read from.
     */
    public CsvReader() {

    }

    /**
     * Constructor taking a Reader to use as the csv source.
     * TODO: should the argument be Reader or a more specific implementation?
     * @param reader A reader containing the csv data to be parsed.
     */
    public CsvReader(Reader reader) {
        this.reader = new BufferedReader(reader);
    }


    /**
     * Class constructor which takes the name of the CVS file to parse.
     * @param file The CVS file to parse.
     * @throws java.io.FileNotFoundException If <code>file</code> is not found.
     */
    public CsvReader(File file) throws FileNotFoundException {
    	if (!file.canRead()) {
    		throw new FileNotFoundException(
    				MsgConstants.Msg.FileNotFound(file.getName()));
    	}
        // TODO: figure out why we are storing the filename as a String.
    	this.filename = file.toString();

    	reader = new BufferedReader(new FileReader(file));
    }

    /**
     * Reads a <code>List</code> of all records in the CSV file.
     * <p>
     * See <code>readNext</code> for additional information.
     * @return <code>List</code> of <code>String</code> arrays representing
     * the contents of the CSV file, where each <code>String</code> array
     * entry in the <code>List</code> holds the elements loaded from each
     * valid row in the CSV file.
     * @throws CsvException <code>CsvFileLoader</code> will attempt to skip
     * the invalid row and proceed.  It will log the error and log that it is
     * attempting to continue.  If the value is set to false, the file loader
     * will throw a <code>CsvException</code>.
     * @throws java.io.IOException Any exception thrown by the <code>Reader</code>.
     */
    public List<String[]> readAll() throws CsvException, IOException {
    	List<String[]> rows = new ArrayList<String[]>();
    	String[] row;

    	while ((row = readNext()) != null) {
    		rows.add(row);
    	}
    	return rows;
    }

//    /**
//     * Reads the first data row in the CSV File.  It skips comments
//     * and blank lines.
//     * <p>
//     * This method can only be called <i>before</i>
//     * a call to readNext() or readAll().  There is no requirement to ever
//     * call this method, however if it is needed, it must be called first.
//     * If it is called later, <code>HeaderRowReadException</code> will be thrown.
//     * <p>
//     * After a call to <code>readHeaderRow</strong>, the reader will be set to
//     * the next row.  A subsequent <code>readNext</code> or <code>readAll</code>
//     * will read entry following that read by <code>readHeaderRow</code>.
//     *
//     * @return A String array of the current row of data.
//     * @throws java.io.IOException Any exception thrown by the <code>Reader</code>.
//     * @throws CsvException if this method is called after
//     * <code>readNext</code> or <code>readAll</code> have been called.  May also be
//     * thrown if unable to read from the <code>File</code>.  The exception's
//     * <code>message</code> will provide additional information.
//     */
//    @SuppressWarnings({"UnusedDeclaration"})
//    public String[] readHeaderRow() throws IOException, CsvException {
//    	String line;
//
//        logger.debug("in readHeaderRow");
//
//        logger.debug("columnCount = " + columnCount);
//    	if (columnCount > 0) {
//            logger.debug("columnCount = " + columnCount);
//    		throw new CsvException("Header Row has already been read");
//    	}
//		while ((line = reader.readLine()) != null) {
//			line = line.trim();
//			logger.debug("line = '" + line + "'");
//			if (isComment(line) || line.isEmpty()) {
//                logger.debug("commented line = '" + line + "'");
//				incrementLineCount();
//			} else if (columnCount < 0) {
//				incrementRowCount();
//				setHeaderRow(line);
//                return headerRow;
//			}
//		}
//		throw new CsvException("Error reading Header Row");
//    }


    /**
     * Reads the next row of data from the CSV file.
     * <p>
     * If this is the first call to this method, returns the first row.
     * <p>
     * <code>readNext</code> skips empty rows (any row which contains
     * only white space) and comments (rows beginning with the
     * <code>'#'</code> <i>hash</i> character in the first column of the
     * line.  The entire line is ignored and the next line is read until
     * a valid line of data is returned.
     * <p>
     * If <code>setSkipErrorLine</code> is set to <code><i>true</i></code>
     * (the default) <code>CsvFileLoader</code> will attempt to skip the
     * invalid row and proceed.  It will log the error and log that it is
     * attempting to continue.  If the value is set to false,
     * <code>readNext</code> will throw a <code>CsvException</code>.
     *
     * @return a <code>String</code> array containing the contents of the
     * line read, split at the point of the delimiter.  Each row of the
     * current line will be placed in order in result, where:
     * <pre>
     * String s[0] = column 1, s[1] = column 2, ... s[<i>n</i>-1] = column <i>n</i>
     * </pre>
     * @throws CsvException <code>CsvFileLoader</code> will attempt to skip
     * the invalid row and proceed.  It will log the error and log that it is
     * attempting to continue.  If the value is set to false, the file loader
     * will throw a <code>CsvException</code>.
     * @throws java.io.IOException Any exception thrown by the <code>Reader</code>.
     */
    public String[] readNext() throws CsvException, IOException {

		String line;

		while ((line = reader.readLine()) != null) {
			line = line.trim();
			logger.debug(">>> [readNext()] >>> current line: " + line);
			if (isComment(line) || line.isEmpty()) {
				logger.debug("currently in comment or blank...");
				incrementLineCount();
			} else {
                logger.debug("processing line");
                incrementRowCount();
                return this.parseLine(line);
            }
		}
		return EMPTY_STRING_ARRAY;
	}

    /**
     * Closes the Reader.
     *
     * <i>This method will catch and handle any exceptions generated
     * by the underlying call to</i> <code>Reader.close</code>
     */
    @SuppressWarnings({"UnusedDeclaration"})
    public void close() {
		try {
			reader.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		reader = null;
	}

    /**
     * Sets the delimiter used for reading the CSV file.
     * <p>
     * Default value is a <code>','</code> <i>comma</i>.
     * @param delimiter the character to be used as the delimeter in the CSV file.
     */
    @SuppressWarnings({"UnusedDeclaration"})
    public void setDelimiter(char delimiter) {
    	this.delimiter = delimiter;
    }

    /**
     * Returns the value of the delimiter used for reading the CSV file.
     * @return the char set as the delimeter.
     */
    @SuppressWarnings({"UnusedDeclaration"})
    public char getDelimiter() {
    	return this.delimiter;
    }

    /**
     * Sets the action to take if an error is encountered while reading
     * a row from the CSV file.
     * <p>
     * If value is set to <strong><code>true</code></strong>, the <code>read</code> method
     * will try to skip the line and proceed with the next line.  It will log
     * the error and log that it is attempting to continue.
     * <p>
     * If value is set to <strong><code>false</code></strong>, the file loader
     * will throw a <code>CsvException</code>.
     * @param skipErrorLines boolean denoting if lines containing errors should be skipped.
     */
    public void setSkipErrorLines(boolean skipErrorLines) {
    	this.skipErrorLines = skipErrorLines;
    }

    /**
     * Returns the value of <code>setSkipErrorLines</code>.
     * <p>
     * See <code>setSkipErrorLines</code> for additional information.
     * @return <code>true</true> or <code>false</code>
     */
    @SuppressWarnings({"UnusedDeclaration"})
    public boolean getSkipErrorLines() {
    	return this.skipErrorLines;
    }

    //-- parse line - the main workhorse.
    private String[] parseLine(String line) throws CsvException, IOException {
        ParseState mode = ParseState.NORMAL;

        List<String> row = new ArrayList<String>();

        //-- stores the chars of the piece we are building up
        StringBuilder workingPiece = new StringBuilder();

        for (int i = 0; i < line.length(); i++) {

        	//-- Case we are the end of a line.
            if (i == line.length() - 1) {


                row.add(this.processLastChar(workingPiece, line.charAt(i), mode));

                //-- set columnCount if we are on the first line.
                if (rowCount == 1) {
                    columnCount = row.size();
                }

                logger.debug("Adding last piece to entry then adding entry to data");

                if (row.size() != columnCount) {
                    if (skipErrorLines) {
                        logger.error("Found bad row in file, trying to continue.");
                        logger.error(MsgConstants.Msg.CvsReadException(
                                filename, lineCount, columnCount, row.size()));
                        return readNext();
                    }
                    throw new CsvException(
                            MsgConstants.Msg.CvsReadException(
                                    filename, lineCount, columnCount, row.size()));
                }


                return row.toArray(new String[columnCount]);
            }
            //-- We are still in a line parsing
            else {
                switch (mode) {
                    // Not quoted or escaped, normal flow.
                    case NORMAL:
                    if (line.charAt(i) == CHAR.BACKSLASH) {
                        mode = ParseState.ESCAPED;
                        break;
                    }
                    if (line.charAt(i) == CHAR.DOUBLE_QUOTE) {
                        mode = ParseState.IN_QUOTE;
                        break;
                    }
                    if (line.charAt(i) == delimiter) {
                        mode = ParseState.NORMAL;
                        logger.debug("Encountered a delimeter after: [" + workingPiece.toString().trim() + "]");

                        row.add(workingPiece.toString().trim());
                        workingPiece = new StringBuilder();
                        break;
                    }

                    workingPiece.append(line.charAt(i));
                    break;

                // We are currently escaped.
                // ** Need to do this rather than a sb.append(++i) because
                // we need to account for the possibility of end of line.
                case ESCAPED:

                    workingPiece.append(line.charAt(i));
                    mode = ParseState.NORMAL;
                    break;

                // Currently in a quote.
                case IN_QUOTE:
                    if (line.charAt(i) == CHAR.DOUBLE_QUOTE) {
                        mode = ParseState.NORMAL;
                        break;
                    }
                    if (line.charAt(i) == CHAR.BACKSLASH) {
                        mode = ParseState.QUOTED_ESCAPED;
                        break;
                    }
                    workingPiece.append(line.charAt(i));
                    break;

                case QUOTED_ESCAPED:
                    mode = ParseState.IN_QUOTE;
                    workingPiece.append(line.charAt(i));
                    break;

                } // switch
            } // if / else
        } // for
        return EMPTY_STRING_ARRAY;
    }


    //-- Process Last Char on the line. (final character of a record)
    private String processLastChar(StringBuilder sb, char c, ParseState mode) {
        if (mode == ParseState.QUOTED_ESCAPED) {
            mode = ParseState.IN_QUOTE;
        }

        switch (mode) {
            // Not quoted or escaped, normal flow.
            case NORMAL:
                if (c == CHAR.BACKSLASH) {
                    // TODO: Come up with CSV Parse Exceptions.
                    logger.debug("LAST CHAR - Error condition... 'Starting Escape on last position of line.'");
                    // IGNORE character
                    break;
                }
                if (c == CHAR.DOUBLE_QUOTE) {
                    logger.debug("LAST CHAR - Error condition... 'Starting Quote on last position of line.'");
                    // IGNORE character
                    break;

                }
                if (c == delimiter) {
                    // TODO: Is this a special case?  This appears to work correctly
                    //       The last field is empty.
                    logger.debug("LAST CHAR - Condition... 'Starting Delimiter on last position of line.'");
                    break;
                }

                sb.append(c);
                break;

            case ESCAPED:
                logger.debug("LAST CHAR - Condition... 'Finishing Escaped char on last position of line.'");
                sb.append(c);

                break;

                // Currently in a quote.
            case IN_QUOTE:
                if (c == CHAR.DOUBLE_QUOTE) {
                    logger.debug("LAST CHAR - Condition... 'Ending Quote on last position of line.'");
                } else if (c == CHAR.BACKSLASH || c == delimiter) {
                    logger.debug("LAST CHAR - Error condition... 'Not Ending quoted text on last position of line,"
                            + " trying end with a backslash or delimiter.'");
                    // IGNORE TODO: Document
                } else {
                    logger.debug("LAST CHAR - Error condition... 'Not Ending quoted text on last position of line.'");
                    // Taking a chance and adding assuming the user stripped a
                    // trailing quote off. TODO: Document
                    sb.append(c);
                }
                break;
        } // switch

        logger.debug("Returning last field in row: [" + sb.toString().trim() + "]");
        return sb.toString().trim();
    }


    //-- See attribute descriptions.
    private void incrementLineCount() {
    	lineCount++;
    }

    //-- See attribute descriptions.
    private void incrementRowCount() {
    	rowCount++;
    	incrementLineCount();
    }

    //-- getDelimiterString - used to convert from char to String
    private String getDelimiterString() {
    	return Character.toString(delimiter);
    }

    //-- Evaluates if the current line is a comment line
    private boolean isComment(String line) {
        if (line.startsWith(COMMENT_CHAR)) {
            logger.debug("Current line is a comment: " + line);
            return true;
        }
        return false;
    }

// -- following field deprecated.  We use the first non-empty row as the
//    row defining the column header.
// /**
//  * The tokens that begin the column definitions in a data file.
//  * If the file contains: <code>[[A, B, C, D</code>
//  * starting in the first column of a row of data, it will
//  * enter that as the column name.  The column name will be 
//  * used as named parameters for when replacing the template outpout
//  * file.
//  */
// private final static String COLUMN_DEF_TOKEN = "[[";



    //---
    //------- Constants used by this class
    //--------------------------------------

    protected final static String COMMENT_CHAR = "#";

    protected final static class CHAR {
        public final static char DOUBLE_QUOTE = 0x22;
        public final static char SINGLE_QUOTE = 0x27;
        public final static char COMMA = 0x2c;
        public final static char BACKSLASH = 0x5c;
        public final static char DEFAULT_DELIMITER = COMMA;
    }


    /**
     * Holds the parse state for parsing a row.
     */
    private enum ParseState {
        NORMAL,
        ESCAPED,
        QUOTED_ESCAPED,
        IN_QUOTE
    }



    //-- A main function for testing
    public final static void main(String[] args)
    		throws CsvException, IOException {
    	File f = new File("c:\\opt\\data\\TEST_DATA2.data");

    	CsvReader reader = new CsvReader(f);
    	reader.setSkipErrorLines(true);
    	String[] line = null;

    	List<String[]> rows = new ArrayList<String[]>();

    	while((line = reader.readNext()) != null) {
    		for (int i = 0; i < line.length; i++) {
    			System.out.print(" [" + line[i] + "]");
    		}
    		rows.add(line);
    		System.out.println("\n");
    	}

    	System.out.println("\n\n\n-----------------------");
    	for (String[] row:rows) {
    		for (int i = 0; i < row.length; i++) {
    			System.out.print(" [" + row[i] + "]");
    		}
    		System.out.print("\n");
    	}
    	System.out.println("-----------------------\n\n");


    	File f2 = new File("c:\\opt\\data\\TEST_DATA1.data");

    	CsvReader reader2 = new CsvReader(f2);

    	List<String[]> rows2 = reader2.readAll();
//    	List<String[]> rows2 = new ArrayList<String[]>();
//    	while((line2 = reader2.readNext()) != null) {
//    		for (int i = 0; i < line2.length; i++) {
//    			System.out.print(" [" + line2[i] + "]");
//    		}
//    		rows2.add(line2);
//    		System.out.println("\n");
//    	}
    	System.out.println("\n\n\n-----------------------");
    	for (String[] row2:rows2) {
    		for (int i = 0; i < row2.length; i++) {
    			System.out.print(" [" + row2[i] + "]");
    		}
    		System.out.print("\n");
    	}
    	System.out.println("-----------------------\n\n");
    }
}
